﻿using System;
using System.Collections.Generic;
using System.Text;
using ShootSearch.Core;
using ShootSearch.Parsers;
using Quartz;
using System.Data;
using System.Windows.Forms;
using System.Text.RegularExpressions;
using System.Collections;
using ShootSearch.Common;
using ShootSearch.FileSystem;
using ShootSearch.Util;
using System.Threading;
using System.Net;
using System.IO;
using ShootSearch.Logger;

namespace ShootSearch.Spiders
{
    public delegate List<string> DownloadUrlCallback(Url url, out short depth);


    public class HttpSpider: ISpiderMan
    {
        public const string PATH_QUEUE = @"Queue\";
        public const string PATH_FAILED = @"Failed\";
        public const string PATH_PROCESSED = @"Processed\";
        public const string PATH_DOWNLOAD = @"Download\";
        public const string PATH_CONFIG = @"Job.config";


        #region 类成员私有变量
        /// <summary>
        /// 配置文件
        /// </summary>
        protected HttpConfig config;

        /// <summary>
        /// 默认编码
        /// </summary>
        protected Encoding encodingDefault;

        /// <summary>
        /// 当前队列
        /// </summary>
        protected IUrlParser urlParser;

        /// <summary>
        /// 文件存储系统
        /// </summary>
        protected IFileSystem fileSaver;

        /// <summary>
        /// 处理失败队列
        /// </summary>
        protected UrlQueue failed;
        /// <summary>
        /// 处理过的队列
        /// </summary>
        protected UrlHashList processed;
        /// <summary>
        /// 未处理队列
        /// </summary>
        protected UrlQueue queue;
 
        /// <summary>
        /// 更新Url列表
        /// </summary>
        protected Hashtable updateList;

        /// <summary>
        /// 完成的线程个数
        /// </summary>
        protected int completeCount = 0;

        /// <summary>
        /// Running thread count
        /// </summary>
        protected int runningThread = 0;


        /// <summary>
        /// Mutex locker
        /// </summary>
        protected AutoResetEvent resetEvent = new AutoResetEvent(false);

        /// <summary>
        /// Url过滤器,这个是决定Url是否进入队列的
        /// </summary>
        protected Regex regexFilter;

        /// <summary>
        /// 忽略的关键字过滤器
        /// </summary>
        protected Regex regexIgnored;

        /// <summary>
        /// 更新的Url过滤器
        /// </summary>
        protected Regex regexUpdate;

        /// <summary>
        /// 更新完成的Url个数
        /// </summary>
        protected int updatedCount;

        /// <summary>
        /// 事件触发参数
        /// </summary>
        //protected EventArgs eventarg;
        /// <summary>
        /// 线程锁
        /// </summary>
        protected object locker = new object();

        /// <summary>
        /// 允许下载的文件类型
        /// </summary>
        protected Hashtable allowContentType;

        /// <summary>
        /// 允许下载的文件类型,有些Web服务器无法正确获取ContentType,那么就检查后缀
        /// </summary>
        protected Regex regexAllowTypes;

        protected long fileSystemMaxlength = 0;
        protected int fileSystemID = 0;
        protected System.Threading.Timer fileSystemTimer;

        protected string contextPath = string.Empty;
        
        #endregion

        #region IInterruptableJob Members

        public void Interrupt()
        {
            if (Status == JobStatus.Running)
                Status = JobStatus.Stopping;   
        }

        #endregion

        #region IJob Members

        public void Execute(JobExecutionContext context)
        {
            if (Status == JobStatus.Unstarted) Status = JobStatus.Running;

            config = context.MergedJobDataMap[Task.JOB_CONFIG] as HttpConfig;
            contextPath = context.MergedJobDataMap[Task.JOB_CONTEXT_PATH] as string;
            contextPath = IO.FormatDirName(contextPath, true);
            contextPath = IO.FormatDirName(contextPath + context.JobDetail.Name, true);
            if (config == null || string.IsNullOrEmpty(contextPath))
            {
                JobExecutionException e2 = new JobExecutionException("Spider configuration error!");
                // Quartz will automatically unschedule
                // all triggers associated with this job
                // so that it does not run again
                e2.UnscheduleAllTriggers = true;
                throw e2;
            }
            else
            {

                
                if (Initialize())
                {
                    LauchDownload();
                    Dispose();
                }
            }

            System.Diagnostics.Debug.WriteLine(context.JobDetail.Name + " Completed!");

            if (Next != null)
            {
                context.Scheduler.ScheduleJob(Next, new SimpleTrigger());
            }

            

        }


        #endregion

        #region ISpiderMan Members

        public JobDetail Next { get; set; }

        public JobStatus Status { get; protected set; }

        public DataTable DetailStatus { get; protected set; }

        public SpiderManConfig Config
        {
            get { return config; }
            set { config = value as HttpConfig; }
        }

        #endregion

        #region IDisposable Members

        public void Dispose()
        {
            if (fileSaver != null)
            {
                fileSaver.Close();
            }
        }

        #endregion

        #region 初始化
        /// <summary>
        /// 初始化
        /// </summary>
        protected bool Initialize()
        {
            //更新Url列表
            this.updateList = new Hashtable();

            //待处理队列和更新队列
            this.queue = new UrlQueue(IO.FormatDirName(contextPath + PATH_QUEUE));
            this.failed = new UrlQueue(IO.FormatDirName(contextPath + PATH_FAILED));

            if (File.Exists(contextPath + PATH_CONFIG))
            {
                config = HttpConfig.Load(contextPath + PATH_CONFIG);
            }


            //如果爬虫的动作是更新或者继续下载，那么需要加载已经处理过的Url队列信息
            if (config.InitConfig.Action != SpiderAction.New)
            {
                this.queue.Load();
                this.failed.Load();
                this.processed = UrlHashList.Load(IO.FormatDirName(contextPath + PATH_PROCESSED));
            }
            else
            {
                //不加载则设置缓存路径
                this.queue.BlockLength = this.config.UrlQueueConfig.UrlBlockLength;
                this.failed.BlockLength = this.config.UrlQueueConfig.UrlBlockLength;
                this.processed = new UrlHashList(this.config.UrlQueueConfig.UrlTableSize);
                this.processed.CachePath = IO.FormatDirName(contextPath + PATH_PROCESSED);

                foreach (string item in config.InitConfig.UrlListStart)
                {
                    //
                    this.queue.EnQueue(new Url(item));
                }
   
            }
            //初始化编码信息

            //获取设置编码信息
            //这个是访问网页时使用的
            if (this.config.InitConfig.Encode.ToLower().EndsWith("default"))
            {
                this.encodingDefault = Encoding.Default;
            }
            else
            {
                try
                {
                    this.encodingDefault = Encoding.GetEncoding(config.InitConfig.Encode);
                }
                catch (Exception ex)
                {
                    this.encodingDefault = Encoding.Default;
                }
            }

            //初始化正则过滤器群
            //config.FilterConfig
            //regexRegular = Tools.CreateRegex(config.FilterConfig.RegexUrlRegular);
            regexFilter = Tools.CreateRegex(config.FilterConfig.RegexEnQueueUrlFilter);
            regexIgnored = Tools.CreateRegex(config.FilterConfig.RegexIgnoredUrlTokens);
            regexUpdate = Tools.CreateRegex(config.FilterConfig.RegexUrlUpdate);
            //config.SaveConfig
            //regexRename = Tools.CreateRegex(config.SaveConfig.RegexIgnoredRename);

            //允许下载的文件类型,有些Web服务器无法正确获取ContentType,那么就检查后缀
            regexAllowTypes = Tools.CreateRegex(config.FilterConfig.ContactToRegex());
            allowContentType = new Hashtable();
            if (config.FilterConfig.FileTypes != null)
            {
                foreach (BinFileDescType filetype in config.FilterConfig.FileTypes)
                {
                    if (!allowContentType.Contains(filetype.MIME))
                    {
                        allowContentType.Add(filetype.MIME, null);
                    }
                }
            }

            //如果线程数出错就设置为默认值5
            if (config.ThreadConfig.ThreadCount <= 0)
            {
                config.ThreadConfig.ThreadCount = 5;
            }
     
            //初始化文件系统
            try
            {
                //反射文件系统
                this.fileSaver = (IFileSystem)config.SaveConfig.FileSystem.GetObject();
                if (config.SaveConfig.FileSystem.Config != null)
                {
                    fileSaver.Config = config.SaveConfig.FileSystem.Config;
                }
                if ((fileSaver != null))
                {
                    fileSaver.FSID = config.SaveConfig.FileSystemID;
                    fileSaver.CreateOpen(IO.FormatDirName(contextPath + PATH_DOWNLOAD));
                    //开启只写模式
                    fileSaver.WriteOnly = true;
                    if (config.SaveConfig.EanbleMultiFileSystem)
                    {
                        if (config.SaveConfig.MultiFileSystemType == MultiFileSystemType.BySzie)
                        {
                            fileSaver.FileSystemWrite += new FileSystemActionEventHandler(fileSaver_FileSystemWrite);  
                            fileSystemMaxlength = config.SaveConfig.MultiFileSystemParameter * 1024 * 1024;
                        }
                        else
                        {
                            fileSystemTimer = new System.Threading.Timer(new TimerCallback(FileSystemChangeIDTimer));
                            fileSystemTimer.Change(config.SaveConfig.MultiFileSystemParameter * 60 * 1000, 0);
                        }
                        fileSystemID = int.Parse(fileSaver.FSID);

                    }
                }
                else
                {
                    return false;
                }

                //反射Url过滤器
                urlParser = config.FilterConfig.UrlParser.GetObject() as IUrlParser;
                if (urlParser == null) return false;
                if (config.FilterConfig.UrlParser.Config != null)
                {
                    urlParser.Config = config.FilterConfig.UrlParser.Config;
                }


            }
            catch (Exception e)
            {

                return false;
                //Log.Error(e);
            }

            return true;
        }

        #endregion

        #region 开始下载
        protected void LauchDownload()
        {
            DownloadUrlCallback downloadUrlCallback = new DownloadUrlCallback(DownloadUrl);
            AsyncCallback parseCallback = new AsyncCallback(ProcessUrls);
            short depth = 0;

            while (Status == JobStatus.Running)
            {
                if (runningThread == 0 && queue.Length == 0)
                {
                    Status = JobStatus.Completing;
                }
                else if (runningThread > 0 && queue.Length == 0)
                {
                    //如果有线程正在下载并且当前的队列为空，那么等待1秒
                    Thread.Sleep(1000);
                }
                else
                {
                    //如果线程数已经达到上限，阻塞线程
                    if (runningThread == config.ThreadConfig.ThreadCount)
                    {
                        resetEvent.WaitOne();
                    }

                    Url url = queue.DeQueue();
                    if (!processed.Contains(url.Address))
                    {
                        processed.Add(url);
                        downloadUrlCallback.BeginInvoke(url,out depth, parseCallback, downloadUrlCallback);
                        runningThread++;
                    }
                }

            }

            while (runningThread > 0)
            {
                Thread.Sleep(1000);
            }

            SaveQueue();
        } 
        #endregion

        #region 两种切换文件系统ID的方式
        /// <summary>
        /// 文件系统写入事件
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        void fileSaver_FileSystemWrite(object sender, ShootSearch.FileSystem.FileSystemEventArgs e)
        {
            if (fileSaver.Length > fileSystemMaxlength)
            {
                fileSaver.FSID = (++fileSystemID).ToString(HttpSaveConfig.FILESYSTEM_ID_FORMAT);
            }
        } 

        /// <summary>
        /// 根据时间来切换文件系统ID
        /// </summary>
        /// <param name="obj"></param>
        private void FileSystemChangeIDTimer(object obj)
        {
            fileSaver.FSID = (++fileSystemID).ToString(HttpSaveConfig.FILESYSTEM_ID_FORMAT);
        }
        #endregion

        #region 下载一个URL
        /// <summary>
        /// 下载一个URL
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        protected List<string> DownloadUrl(Url url, out short depth)
        {
            string page = string.Empty;
            depth = url.Depth;
            List<string> urls = new List<string>();
          
            Uri uri = null;
            HttpWebRequest request = null;
            HttpWebResponse response = null;
            Stream stream = null;
            //防止传入的Url出错
            try
            {
                uri = new Uri(url.Address);

                #region 创建请求信息
                //创建请求信息
                request = WebRequest.Create(uri) as HttpWebRequest;
                request.Accept = config.ThreadConfig.HttpAccept;
                request.Method = config.ThreadConfig.HttpMethod;
                request.Referer = config.ThreadConfig.HttpReferer;
                request.KeepAlive = true;
                request.UserAgent = config.ThreadConfig.HttpUserAgent;
                request.Headers["Accept-Language"] = config.ThreadConfig.HttpAcceptLanguage;
                request.Headers["Accept-Encoding"] = config.ThreadConfig.HttpAcceptEncoding;
                request.Headers["Accept-Charset"] = config.ThreadConfig.HttpAcceptCharset;
                request.Timeout = config.ThreadConfig.TimeOut * 1000;
                #endregion

                //发送请求
                response = request.GetResponse() as HttpWebResponse;

                

                if (response != null)
                    stream = response.GetResponseStream();

                //检查长度是否在规定的范围内
                if (CheckLength(response, stream))
                {
                    #region 检查 ContentType
                    //检查 ContentType 如果为 1空 或者 2文本 或者 3是允许下载的类型
                    if (string.IsNullOrEmpty(response.ContentType) ||
                            response.ContentType.StartsWith("text") ||
                            allowContentType.Contains(response.ContentType.ToLower()) ||
                            (regexAllowTypes != null && regexAllowTypes.IsMatch(url.Address)))
                    {
                        System.Diagnostics.Debug.WriteLine("Downloading[" + Thread.CurrentThread.ManagedThreadId + "]: " + url.Address);

                        //读取网络流到内存
                        MemoryStream memStream = new MemoryStream();
                        IO.Pump(stream, memStream, config.SaveConfig.BufferChunk);
                        memStream.Seek(0, SeekOrigin.Begin);
                        //网络流读取完毕
                        stream.Close();

                        #region 检查是否是压缩类型
                        //检查是否是压缩类型
                        byte[] bytes = new byte[0];
                        if (response.ContentEncoding.ToLower() == "gzip")
                        {
                            MemoryStream memOriginalStream = new MemoryStream();
                            IO.Decompress(memStream, memOriginalStream);
                            //Get byte array
                            bytes = new byte[(int)memOriginalStream.Length];
                            memOriginalStream.Seek(0, SeekOrigin.Begin);
                            memOriginalStream.Read(bytes, 0, bytes.Length);

                        }
                        else
                        {
                            //Get byte array
                            bytes = new byte[(int)memStream.Length];
                            memStream.Read(bytes, 0, bytes.Length);
                        }
                        #endregion

                        #region 保存，解析文件
                        //再次检查类型问题
                        if (response.ContentType.StartsWith("text"))
                        {
                            Encoding encoding = WebPage.GetEncoding(response.ContentType, null);
                            if (encoding == null) encoding = WebPage.TestPageEncoding(bytes, encodingDefault);
                            page = encoding.GetString(bytes);
                            //保存文本
                            fileSaver.WriteText(url.Address, page);

                            //判断是否进行下一步的解析工作
                            if (config.FilterConfig.UrlMaxDepth <= 0
                                || url.Depth < config.FilterConfig.UrlMaxDepth)
                            {
                                urls = urlParser.ParsePage(url.Address, page);
                                depth++;
                            }


                        }
                        else
                        {
                            //保存二进制文件
                            fileSaver.Write(url.Address, response.ContentType + "|" + url.Address, bytes);
                        }
                        #endregion


                    }
                    #endregion
                }

                //关闭响应对象
                response.Close();


            }
            catch (Exception ex)
            {
                Console.WriteLine(ex);
                //下载出错
                failed.EnQueue(url);

            }

            return urls;
        } 
        #endregion

        #region 检查文件长度是否在规定的范围内
        /// <summary>
        /// 检查文件长度是否在规定的范围内
        /// </summary>
        /// <param name="response">HTTP响应对象</param>
        /// <param name="stream"></param>
        /// <returns></returns>
        protected bool CheckLength(HttpWebResponse response, Stream stream)
        {
            //检查stream是否为空
            bool success = stream != null;
            if (!success ||
                config.FilterConfig.MaxFileLength == 0 && config.FilterConfig.MinFileLength == 0) return success;

            //如果用户有设置长度限制则先尝试读取头部信息里面的长度
            long flength = -1;

            try
            {
                flength = Convert.ToInt64(response.Headers["content-length"]);
            }
            catch (FormatException e)
            {
                System.Diagnostics.Debug.WriteLine(e.Message);
                //Log.Warn("无法读取长度!" + e.ToString());
                flength = -1;
            }


            if (flength == -1)
            {
                try
                {
                    if(stream.CanSeek)
                        //尝试读取流的长度
                        flength = stream.Length;
                }
                catch (FormatException e)
                {
                    //Log.Warn("无法读取长度!" + e.ToString());
                    flength = -1;
                }

            }


            //如果文件长度可以判断先尝试判断长度
            if (flength != -1)
            {
                if ((config.FilterConfig.MaxFileLength > 0)
                    && (flength > config.FilterConfig.MaxFileLength))
                {
                    success = false;
                }
                if ((config.FilterConfig.MinFileLength > 0)
                    && (flength < config.FilterConfig.MaxFileLength))
                {
                    success = false;
                }
            }

            return success;
        } 
        #endregion

        #region 处理下一级的Url队列
        /// <summary>
        /// 处理下一级的Url队列
        /// </summary>
        /// <param name="result"></param>
        protected void ProcessUrls(IAsyncResult result)
        {
            short depth = 0;
            DownloadUrlCallback downloadUrlCallback = result.AsyncState as DownloadUrlCallback;
            List<string> urls = downloadUrlCallback.EndInvoke(out depth,result);

            for (int i = 0; i < urls.Count; i++)
            {
                //如果只下载静态页面
                if (config.FilterConfig.StaticOnly && urls[i].IndexOf("?") > -1) continue;
                //包含过滤词
                if (regexIgnored != null && regexIgnored.IsMatch(urls[i])) continue;
                //不符合入队条件
                if (regexFilter != null && !regexFilter.IsMatch(urls[i])) continue;
                ////检查是否符合更新列表
                //if (regexUpdate != null && regexUpdate.IsMatch(urls[i].Address) 
                //    && !updateList.Contains(urls[i].Address ))
                //{
                //    updateList.Add(urls[i].Address,"");
                //}

                queue.EnQueue(new Url(depth, urls[i]));
            }

            runningThread--;
            resetEvent.Set();
        } 
        #endregion

        #region SaveQueue
        /// <summary>
        /// 保存队列
        /// </summary>
        protected void SaveQueue()
        {
            if (Status == JobStatus.Stopping)
            {
                config.InitConfig.Action = SpiderAction.Resume;
            }
            else if (Status == JobStatus.Completing)
            {
                config.InitConfig.Action = SpiderAction.Update;
            }

            //使用线程锁,可以在运行过程中保存
            lock (locker)
            {
                //保存配置进度信息
                this.config.Save(contextPath + PATH_CONFIG);
                //未处理队列
                this.queue.Save();
                //处理失败队列
                this.failed.Save();
                //处理过Url队列
                this.processed.Save();
                if (fileSaver != null)
                {
                    fileSaver.Flush();
                }
            }
        } 
        #endregion

        public HttpSpider()
        {
            config = new HttpConfig();
        }

        
    }
}
